package cn.itcast.webmagic.test;

import org.jsoup.Jsoup;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;
import us.codecraft.webmagic.scheduler.BloomFilterDuplicateRemover;
import us.codecraft.webmagic.scheduler.QueueScheduler;
import us.codecraft.webmagic.scheduler.Scheduler;

public class JobProcessor implements PageProcessor {

    //  解析页面
    @Override
    public void process(Page page) {
//        解析返回的数据，放入到ResultItems中
//        css选择器
//        page.putField("div", page.getHtml().css("div.list ul li a").all());
//
////  xpath选择器
//        page.putField("div2", page.getHtml().xpath("//ul[@id=navitems-group2]/li"));
//
//// 正则表达式
//        page.putField("div3", page.getHtml().css("ul.fr li a").regex(".*京东.*").all());
//
////        处理结果的API,
//        page.putField("div4", page.getHtml().css("ul.fr li a").regex(".*京东.*").get());//返回第一个一个
//        page.putField("div5", page.getHtml().css("ul.fr li a").regex(".*京东.*").toString());
//获取连接
//        page.addTargetRequests(page.getHtml().css("div#navitems-2014").links().regex(".*l$").all());//.links().regex(".*l$").顺序不能变
////        page.addTargetRequests(page.getHtml().css("div#navitems-2014").regex(".*l$").links().all());
//        page.putField("url", page.getHtml().css("div#navitems-2014 a").all());
        page.addTargetRequest("https://search.51job.com/list/000000,000000,0121%252c0154%252c0107%252c7202%252c7201,01%252c32%252c38,9,99,%25E5%2589%258D%25E7%25A8%258B%25E6%2597%25A0%25E5%25BF%25A7%2520-%25E4%25BB%25A3%25E6%258B%259B%25E8%2581%2598%2520-51Job%25E6%258B%259B%25E8%2581%2598%25E4%25BC%259A,1,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=");
        page.addTargetRequest("https://search.51job.com/list/000000,000000,0121%252c0154%252c0107%252c7202%252c7201,01%252c32%252c38,9,99,%25E5%2589%258D%25E7%25A8%258B%25E6%2597%25A0%25E5%25BF%25A7%2520-%25E4%25BB%25A3%25E6%258B%259B%25E8%2581%2598%2520-51Job%25E6%258B%259B%25E8%2581%2598%25E4%25BC%259A,1,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=");
        page.addTargetRequest("https://search.51job.com/list/000000,000000,0121%252c0154%252c0107%252c7202%252c7201,01%252c32%252c38,9,99,%25E5%2589%258D%25E7%25A8%258B%25E6%2597%25A0%25E5%25BF%25A7%2520-%25E4%25BB%25A3%25E6%258B%259B%25E8%2581%2598%2520-51Job%25E6%258B%259B%25E8%2581%2598%25E4%25BC%259A,1,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=");

        page.putField("div6", Jsoup.parse(page.getHtml().css("div.rollBox h3").nodes().toString()).text());
    }

    private Site site = Site.me()
            .setCharset("gbk") //设置编码,一般不是utf8就是gbk
            .setTimeOut(10000)  //设置超时时间
            .setRetrySleepTime(3000)  //设置重试的间隔时间
            .setSleepTime(3)      //设置重试次数
            ;//对设置配置

    @Override
    public Site getSite() {
        return site;
    }

    //主函数，执行入口
    public static void main(String[] args) {
        Spider spider = Spider.create(new JobProcessor())
                .addUrl("https://search.51job.com/list/000000,000000,0121%252c0154%252c0107%252c7202%252c7201,01%252c32%252c38,9,99,%25E5%2589%258D%25E7%25A8%258B%25E6%2597%25A0%25E5%25BF%25A7%2520-%25E4%25BB%25A3%25E6%258B%259B%25E8%2581%2598%2520-51Job%25E6%258B%259B%25E8%2581%2598%25E4%25BC%259A,1,1.html?lang=c&postchannel=0000&workyear=99&cotype=99&degreefrom=99&jobterm=99&companysize=99&ord_field=0&dibiaoid=0&line=&welfare=")//设置爬取页面https://www.jd.com/allSort.aspx
//                .addPipeline(new FilePipeline("D:\\maven\\result"))//以文件的格式输出到电脑指定文件夹下
                .thread(5)//设置线程
                .setScheduler(new QueueScheduler().setDuplicateRemover(new BloomFilterDuplicateRemover(100000)))//设置布隆去重过滤器对10000进行去重
                ;


        Scheduler scheduler = spider.getScheduler();


        spider.run();//执行
    }
}
